{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracies: [ 0.95221027  0.95454545  0.96172249  0.96052632  0.95209581]\n",
      "Mean accuracy: 0.956220068309\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model.logistic import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df = pd.read_csv('./sms.csv')\n",
    "X_train_raw, X_test_raw, y_train, y_test = train_test_split(df['message'], df['label'], random_state=11)\n",
    "vectorizer = TfidfVectorizer()\n",
    "X_train = vectorizer.fit_transform(X_train_raw)\n",
    "X_test = vectorizer.transform(X_test_raw)\n",
    "classifier = LogisticRegression()\n",
    "classifier.fit(X_train, y_train)\n",
    "scores = cross_val_score(classifier, X_train, y_train, cv=5)\n",
    "print('Accuracies: %s' % scores)\n",
    "print('Mean accuracy: %s' % np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.992542742398\n",
      "Recall: 0.683605030275\n"
     ]
    }
   ],
   "source": [
    "precisions = cross_val_score(classifier, X_train, y_train, cv=5, scoring='precision')\n",
    "print('Precision: %s' % np.mean(precisions))\n",
    "recalls = cross_val_score(classifier, X_train, y_train, cv=5, scoring='recall')\n",
    "print('Recall: %s' % np.mean(recalls))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 score: 0.809067846627\n"
     ]
    }
   ],
   "source": [
    "f1s = cross_val_score(classifier, X_train, y_train, cv=5, scoring='f1')\n",
    "print('F1 score: %s' % np.mean(f1s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu8VXP+x/HXp3uUUmGoqBHSTaqRDEIuieQyUi4j0wjj\nGsNk8HMZxiQjhlByHVRkKAbFJPeoCJXLpBqV6KLodtLpfH5/fNep7TiX3Tln77X3Pu/n43EeZ++1\n1t77s9fZZ33297I+y9wdERGRklSLOwAREclsShQiIlIqJQoRESmVEoWIiJRKiUJEREqlRCEiIqVS\nopCkmdkZZjY57jgyiZmtNbNfxvC6LczMzaxGul87FcxsjpkdVo7H6TOZBkoUWcrMFprZhuhA9Y2Z\nPWJm9VL5mu7+hLsfncrXSGRmB5nZFDNbY2bfm9nzZtYmXa9fTDxTzez3icvcvZ67z0/R6+1tZk+b\n2Yro/X9sZpebWfVUvF55RQmrVUWew93buvvUMl7nZ8kx3Z/JqkqJIrv1dvd6QEdgf+DqmOMpl+K+\nFZtZN2AyMAHYDWgJfAS8nYpv8Jn2zdzM9gTeAxYB7d29AXAq0AWoX8mvFdt7z7T9LiVwd/1k4Q+w\nEDgy4f5twL8T7tcGbge+Ar4F7gfqJqzvA8wCfgC+BHpGyxsADwJLgSXAzUD1aN0A4K3o9n3A7UVi\nmgBcHt3eDXgGWA4sAC5J2O4GYDzwePT6vy/m/b0J3FvM8peAx6LbhwGLgT8DK6J9ckYy+yDhsX8C\nvgH+CewIvBDFvCq63Sza/hZgM5AHrAXuiZY70Cq6/QgwAvg3sIZwoN8zIZ6jgc+B74F7gdeLe+/R\nto8n/j2LWd8ieu2zo/e3ArgmYf0BwLvA6uhveQ9QK2G9AxcC/wUWRMvuIiSmH4CZwCEJ21eP9vOX\n0XubCTQH3oiea120X06Ltj+e8PlaDbwDdCjy2f0T8DGwEahBwuc5in1GFMe3wB3R8q+i11ob/XQj\n4TMZbdMWeAX4Lnrsn+P+X82Fn9gD0E85/3A//cdqBnwC3JWwfjgwEWhE+Ab6PHBrtO6A6GB1FKFV\n2RRoHa17FhgJbA/sDLwPnBet2/JPCRwaHVQsur8jsIGQIKpFB5L/A2oBvwTmA8dE294AbAJOjLat\nW+S9bUc4KB9ezPs+B1ga3T4MyAfuICSF7tEBa58k9kHhY4dGj60LNAZOiV6/PvA08FzCa0+lyIGd\nnyeKldH+rQE8AYyN1jWJDnwnR+sujfZBSYniG+CcUv7+LaLXfiCKfT/CQXffaH1n4MDotVoAnwKX\nFYn7lWjfFCbPM6N9UAO4IoqhTrTuSsJnbB/AotdrXHQfRPf3B5YBXQkJ5mzC57V2wmd3FiHR1E1Y\nVvh5fhc4K7pdDziwyHuukfBaA9j6maxPSIpXAHWi+13j/l/NhZ/YA9BPOf9w4R9rLeHbnQP/ARpG\n64xwwEz8NtuNrd8cRwLDi3nOXaKDTWLLoz/wWnQ78Z/SCN/wDo3unwtMiW53Bb4q8txXAw9Ht28A\n3ijlvTWL3lPrYtb1BDZFtw8jHOy3T1j/FHBdEvvgMODHwgNhCXF0BFYl3J9K2YlidMK6XsBn0e3f\nAu8mrDNCoi0pUWwiauWVsL7woNksYdn7QL8Str8MeLZI3EeU8RlbBewX3f4c6FPCdkUTxX3AX4ps\n8znQPeGz+7tiPs+FieIN4EagSQnvuaRE0R/4MJX/d1X1R/2D2e1Ed3/VzLoDTxK+ta4GdiJ8K55p\nZoXbGuHbHYRvci8W83x7ADWBpQmPq0Y4oP2Eu7uZjSX8c74BnE7oLil8nt3MbHXCQ6oTupMK/ew5\nE6wCCoBdgc+KrNuV0M2yZVt3X5dw/3+EVk1Z+wBgubvnbVlpth2hFdKT0EICqG9m1d19cynxJvom\n4fZ6wjdiopi2vOdo/y0u5XlWEt5ruV7PzPYmtLS6EPZDDUIrL9FP/gZm9kdgYBSrAzsQPlMQPjNf\nJhEPhL//2WZ2ccKyWtHzFvvaRQwEbgI+M7MFwI3u/kISr7stMco20GB2DnD31wnfZm+PFq0gdAO1\ndfeG0U8DDwPfEP5J9yzmqRYRWhRNEh63g7u3LeGlxwC/MbM9CK2IZxKeZ0HCczR09/ru3isx7FLe\nzzpC98OpxazuS2g9FdrRzLZPuL878HUS+6C4GK4gdK10dfcdCN1rEBJMqTEnYSmhpRSeMGSvZiVv\nzquEbrDyuo+QZPeK3suf2fo+Cm15P2Z2CHAVYf/u6O4NCd2ThY8p6TNTnEXALUX+/tu5+5jiXrso\nd/+vu/cndH0OBcZHf+Oy9v8iQjenVDIlitxxJ3CUme3n7gWEvuvhZrYzgJk1NbNjom0fBM4xsx5m\nVi1a19rdlxJmGv3dzHaI1u0ZtVh+xt0/JByQRwOT3L2wBfE+sMbM/mRmdc2supm1M7NfbcP7GUL4\nVnqJmdU3sx3N7GZC99GNRba90cxqRQe744Gnk9gHxalPSC6rzawRcH2R9d9S/gPRv4H2ZnZiNNPn\nQuAXpWx/PXCQmQ0zs19E8bcys8fNrGESr1efMCay1sxaAxcksX0+YSC/hpn9H6FFUWg08Bcz28uC\nDmbWOFpXdL88AJxvZl2jbbc3s+PMLKnZWmZ2ppntFP0NCz9TBVFsBZT8N3gB2NXMLjOz2tHnpmsy\nrymlU6LIEe6+HHiMMIAMYVbJPGCamf1A+Ia6T7Tt+4RB4eGEb42vE7oLIPSl1wLmErqAxlN6F8iT\nwJHR78JYNhMO2B0JM54Kk0mDbXg/bwHHEAZ/lxK6lPYHDnb3/yZs+k0U59eEwePz3b2wu6rEfVCC\nOwkDwyuAacDLRdbfRWhBrTKzfyT7XqL3s4LQQrqN0K3UhjCzZ2MJ239JSIotgDlm9j2hxTaDMC5V\nlj8SugPXEA7c48rYfhLh/X5B2Nd5/LR76A7C+M9kQgJ6kLCvIIw5PWpmq82sr7vPIIxZ3UP428wj\njCUkqyfhPa8l7PN+7r7B3dcTZp+9Hb3WgYkPcvc1hAkavQmfi/8Ch2/D60oJCmesiGSd6Ezex929\ntC6cjGRm1QjTc89w99fijkekNGpRiKSJmR1jZg3NrDZbxwymxRyWSJlSlijM7CEzW2Zms0tYb2b2\nDzObF5Um6JSqWEQyRDfCrJwVhO6RE919Q7whiZQtZV1PZnYoYZ7/Y+7erpj1vYCLCXPNuxJOFtPA\nk4hIhklZi8Ld3yCcRl+SPoQk4u4+DWhoZsnMGxcRkTSK84S7pvx0VsXiaNnSohua2SBgEMD222/f\nuXXr1mkJUESS9/nnsHYt1EtpDWPZFtU9H8PJt5qsXTtzhbvvVJ7nyYozs919FDAKoEuXLj5jxoyY\nI5JsNGoUPPlk2dtJ+VSvDt27w9SpcUciuMO4cXDxxeGPMn48Zva/8j5dnIliCeGU+0LNomUi2yTZ\nBPD66+F392JPH5SK6tgRTj897iiEJUvgggvg+efhgAPghhsq/JRxJoqJwEVRvaCuwPfRmcGSJrny\nDTvZBNC9eziQDRqU+phEYjF5Mpx6KmzaBHfcAZdcEpp6FZSyRGFmYwgVOptExc+uJxScw93vJxSl\n60U4a3M94UzhnJZpB+Zc+YatBCBVnjuYQbt2cPjh8Pe/w57JluYqW9admZ1tYxSJySETD8w6wIpk\nsc2b4c47YcqU0NVUreSJrGY20927lOdlsmIwO5s9+WRIEN2765uviFSi2bPhd7+D6dOhd29Ytw7q\nV+pVcrdQoiiHbelCmjVLM0FEpBJt3Ai33gp//Ss0bAhjx0LfvqHrKUWUKJJQNDFsSxeSZoKISKVa\ntw7uuw9OOw2GD4cmTcp+TAUpUZRh1Cg477xwuzAxqAtJRNJq3Tq4/3649FJo1Ch0O+1UrnPnykWJ\nIlJSd1Jh62HkSCUGEYnBlClw7rkwfz60bw9HH53WJAEqM77Fk0+G8YSiundXkhCRGKxeHRJEjx5h\nNtPUqSFJxKBKtygSWxGzZoXxBA06i0hGOO44mDYNrroqnF1dt26ZD0mVKpMoiutaShyU1qCziMRu\n+fIwxbVOHfjb30Jy6FKuUx8qVc4mimRmKmlQWkQygns4YF16aajT9Je/wCGHxB3VFjmXKAoTRNHE\noKQgIhlp0SI4/3x48UU48EDo3z/uiH4m5xJF4aC0EoOIZLynnoLf/35rKY6LLqqUIn6VLecSBWhQ\nWkSyRPPm0K1bOEeiZcu4oylRTiYKEZGMlJ8fzqZetgyGDQtJYtKkuKMqU06dRzFq1NaxCRGRjPLR\nR2EM4qqrwslzmzfHHVHSsr5FUVwZb01zFZGMsXEj3HxzmO7aqBE8/TScckpKi/hVtqxOFEXrMGkA\nW0Qyzrx5MHRoODjdcQc0bhx3RNssqxNFYUtCJTZEJKOsXQsTJsAZZ0DbtvDZZ/DLX8YdVbll7RhF\n4XhE9+5KEiKSQV55JRTvO+ss+PTTsCyLkwRkaaJI7HLSeISIZIRVq2DgwFC4r1at8E12333jjqpS\nZGXXk7qcRCSjbNoEv/oVLFwIV18N//d/oV5TjsjKRAHqchKRDLBqVbgcac2acNNN0Lo1dOoUd1SV\nLiu7nkREYuUOjz0Ge+4J48aFZaefnpNJApQoRES2zf/+B8ceC2efHcYgOnaMO6KUU6IQEUnWQw9B\nu3bw1ltw993w5puhuynHZe0YhYhI2hUUwEEHhZk0LVrEHU3aZF2LYvly1XMSkTTZtAluvTW0JCBM\nf3355SqVJCALE8V334XfOn9CRFLqww+ha1f485/h7bfDMrOsqtFUWbIuUYCmxopICuXlheTwq1/B\n11/DM8/Agw/GHVWssjJRiIikzCuvhO6m3/42lOA4+eS4I4qdBrNFRNauhWnT4Mgj4fjj4YMPYP/9\n444qY6hFISJV26RJocJrnz6wcmUYg1CS+AklChGpmr77DgYMgJ49YbvtYPLkrLxWRDqo60lEqp6V\nK0MrYsUKuOYauPbanCriV9mUKESk6tiwAerWDS2HSy6BXr2qRAmOilLXk4jkPnd4+GHYYw+YNSss\n+/OflSSSlNJEYWY9zexzM5tnZkOKWb+7mb1mZh+a2cdm1iuV8YhIFbRgQbiY0O9+F+oy1asXd0RZ\nJ2WJwsyqAyOAY4E2QH8za1Nks2uBp9x9f6AfcG+q4hGRKuiee0IRv2nT4N57YepUaNUq7qiyTipb\nFAcA89x9vrv/CIwF+hTZxoEdotsNgK9TGI+IVDXz5oVSDnPmwAUXQDX1tpdHKgezmwKLEu4vBroW\n2eYGYLKZXQxsDxxZ3BOZ2SBgEEDt2h0qPVARyRGbNsFtt4XkcPDBMGwY1KhRJeszVaa402t/4BF3\nbwb0Av5pZj+Lyd1HuXsXd+9Ss2bNtAcpIllg5sxQn+naa+H558OymjWVJCpBKhPFEqB5wv1m0bJE\nA4GnANz9XaAO0CSFMYlIrtmwAYYMCZVely2DZ5+FoUPjjiqnpDJRTAf2MrOWZlaLMFg9scg2XwE9\nAMxsX0KiWJ7CmEQk19x3X0gMAwbA3Llw4olxR5RzUjZG4e75ZnYRMAmoDjzk7nPM7CZghrtPBK4A\nHjCzwYSB7QHu7qmKSURyxA8/hGmv++0HF14IXbrAoYfGHVXOsmw7Ltev38U7d57B1KlxRyIisXjx\nRTj//DCD6b//DeMQUiYzm+nuXcrz2LgHs0VEkrNiBZx1Fhx3HNSvD+PGKUmkiWo9iUjm+/xzOOQQ\nWLUKrr8err4aateOO6oqQ4lCRDLX5s1QvXo4m/rEE+Hii6F9+7ijqnLU9SQimccdRo+GffcNXU7V\nq8OoUUoSMVGiEJHMMn9+uCTpuefCbrvB+vVxR1TlKVGISGYoKIDhw0MRv+nT4f77YcoU2H33uCOr\n8rJujGLt2rgjEJGUMIMXXoAjjghJolmzuCOSSNYlCoDTT487AhGpFD/+GAr3/fa30Lw5TJgA22+v\n+kwZJuu6nurVg0GD4o5CRCps+vRwRvW118L48WFZvXpKEhko6xKFiGS59evhyivhwAPhu+9g4kQY\nPDjuqKQUShQikl5/+hPcfnuY1TRnDvTuHXdEUoasrPW0Zs2MuMMQkW3x/fewZk0YoF66FD77DA4/\nPO6oqhTVehKRzPXCC9C2bRiwBth1VyWJLKNEISKpsXx5mKLYuzfsuCP87W9xRyTllJXTY0Ukw737\nLpxwQuhyuvHGcAW6WrXijkrKSYlCRCqPe5jeus8+0K0b/PWv4UxryWrqehKRiisogJEjw9jDpk3Q\nqFGY9qokkROUKESkYubNgx49wlXnqlcP3U2SU5QoRKR88vPD+RDt28MHH8ADD8Crr0KTJnFHJpVM\nYxQiUj4bN8KIEXD00XDvvdC0adwRSYqoRSEiydu4MZQCz8sLxfveew+ee05JIscpUYhIct57Dzp3\nhssvD1VeAXbeWUX8qgAlChEp3bp1ITl06xYGqv/9bzjttLijkjTSGIWIlK5/f3j+ebjggnB29Q47\nxB2RpJmKAorIz61eHaa61q8PM2aEVkX37nFHJRWgooAiUnkmTIA2bUI5cAgXF1KSqNKUKEQk+Pbb\nMPZw4olhkHrgwLgjkgyhRCEi8PLLoRXx3HNw883hMqWdO8cdlWQIDWaLCDRvHs6wvvfekDBEEqhF\nIVIVFRTAffeF+kwQLiw0daqShBRLiUKkqvniCzjsMPjDH+DLL8NZ1iKlUKIQqSry8+G222C//eCT\nT+Chh2DyZKhTJ+7IJMNpjEKkqliyJFxt7thjQzG/XXeNOyLJEmpRiOSyjRvh0UfDlef22AM+/hie\neUZJQrZJShOFmfU0s8/NbJ6ZDSlhm75mNtfM5pjZk6mMR6RKeecd6NgRBgwI17AG2HNPFfGTbZay\nRGFm1YERwLFAG6C/mbUpss1ewNXAr929LXBZquIRqTLWroVLL4WDD4b168M5EgcdFHdUksVSOUZx\nADDP3ecDmNlYoA8wN2Gbc4ER7r4KwN2XpTAekdznDoceCrNmwYUXwl//Guo1iVRAKhNFU2BRwv3F\nQNci2+wNYGZvA9WBG9z95aJPZGaDgEEAtWt3SEmwIllt9epQ1bVaNbjuOthpp9CiEKkEcQ9m1wD2\nAg4D+gMPmFnDohu5+yh37+LuXWrWrJnmEEUy3LPPwr77wsiR4f5JJylJSKVKZaJYAjRPuN8sWpZo\nMTDR3Te5+wLgC0LiEJGyfPMNnHoqnHwy/OIX0LVog12kcqQyUUwH9jKzlmZWC+gHTCyyzXOE1gRm\n1oTQFTU/hTGJ5Ibx40O5jeefD+MQ778PnTrFHZXkqJSNUbh7vpldBEwijD885O5zzOwmYIa7T4zW\nHW1mc4HNwJXuvjJVMYnkjGrVQnfTgw9C69ZxRyM5Tle4E8kGBQWhsmt+Plx22dZl1eIeZpRsoSvc\nieSyzz4LU14vvjhUeC38cqckIWmiT5pIptq0KYw/7LcfzJ0bSnE8+6zOrJa0U6IQyVTvvQfXXAMn\nnACffgq//a2ShMRCiUIkk+TlwaRJ4fbBB8PMmfD007DLLvHGJVVaqbOezOzy0ta7+x2VG45IFfb2\n2zBwIMybFy4otMcemvIqGaGsFkX9Mn5EpKLWrAkD1YccEsqCv/RSSBIiGaLUFoW735iuQESqpPXr\nQynwBQtCsrjlFqhXL+6oRH6irK6nf5S23t0vqdxwRKqIDRugbl3Ybju46KJQfkOlwCVDlXVm9sy0\nRCFSVbiHK8xdfDGMGxfOjxg8OO6oREpVVtfTo+kKRCTnLV0arhHx7LPQuTPsuGPcEYkkJalaT2a2\nE/AnwpXq6hQud/cjUhSXSG557LFw1bm8PLjtttCKqJHKy8GIVJ5kz6N4AvgUaAncCCwkVIcVkWQs\nXAgdOsBHH8GVVypJSFZJqihgVEyqs5l97O4domXT3f1XKY+wCBUFlKyweTOMGAGtWkGvXqGYX7Vq\nqs8ksUlHUcBN0e+lZnacme0PNCrPC4rkvE8/DedEXHop/OtfYVmNGkoSkrWSbf/ebGYNgCuAu4Ed\nAE3VEEm0aRMMHQp/+Us4F+Kf/4Qzzog7KpEKSypRuPsL0c3vgcNTF45IFnviCbjuOujbF+6+G3be\nOe6IRCpFUm1hM3vUzBom3N/RzB5KXVgiWWLDBvjgg3D7rLPg1VfD+RFKEpJDku007eDuqwvvuPsq\nYP/UhCSSJV5/PcxkOuYYWLsWqleHHj3ijkqk0iWbKKqZ2Zazg8ysESm83rZIRvvhB7jgAjjssHA5\n0nHjVJ9JclqyB/u/A++a2dPR/VOBW1ITkkgGW7wYunWDr7+Gyy+Hm26C7bePOyqRlEp2MPsxM5sB\nFJ6JfbK7z01dWCIZZvPm0LXUtCn06RPGI7p2jTsqkbTYlondjYB17n4PsNzMWqYoJpHM4Q5PPQX7\n7ANffRUuRXrPPUoSUqUkO+vpekKtp6ujRTWBx1MVlEhG+PprOOkkOO20UMAvLy/uiERikWyL4iTg\nBGAdgLt/ja5wJ7nKHUaPhjZtwvWrhw2Dd9+FvfeOOzKRWCQ7mP2ju7uZOYCZafROcpcZvPxyuPLc\n6NGhXpNIFZZsonjKzEYCDc3sXOB3wOjUhSWSZps3wz/+AcceC61bwyOPhKvPqT6TSNKznm43s6OA\nH4B9gP9z91dSGplIusyeDQMHwvvvw8qVcPPNOi9CJEHSJ81FieEVADOrZmZnuPsTKYtMJNV+/BFu\nvRVuuQUaNIAnn4R+/eKOSiTjlNquNrMdzOxqM7vHzI624CJgPtA3PSGKpMgtt8ANN8Cpp8LcudC/\nfxifEJGfKPXCRWY2AVgFvAv0AHYGDLjU3WelJcIidOEiqZD162HZMmjRAlavhrffhuOOizsqkZSr\nyIWLyup6+qW7t49eZDSwFNjd3TWhXLLP1Knw+99D/fowcyY0bKgkIZKEsqZ0FF7ZDnffDCxWkpCs\n8/33cN55cHh0KZU77tBsJpFtUFaLYj8z+yG6bUDd6L4B7u47pDQ6kYr65JMw5XXpUvjjH+HGG8O0\nVxFJWqmJwt2rpysQkUrlHgamf/lL6NwZrrkGDjgg7qhEspLa35Jb3MM011//Olx9bvvtYcIEJQmR\nCkhpojCznmb2uZnNM7MhpWx3ipm5mZVrRF4ECNeKOOEEOOOMcEGhlSvjjkgkJ6QsUZhZdWAEcCzQ\nBuhvZm2K2a4+cCnwXqpikRxXUAAjR4YiflOmwPDhYdprs2ZxRyaSE1LZojgAmOfu8939R2As0KeY\n7f4CDAU0m0rKJz8fRowI3UuffAKXXRYuMiQilSKViaIpsCjh/uJo2RZm1glo7u7/Lu2JzGyQmc0w\nsxmbNm0qbVOpKvLz4e67w9TXWrXg1VfhlVfC4LWIVKrYBrPNrBpwB3BFWdu6+yh37+LuXWrWrJn6\n4CSzffxxuG71JZfAE1G5sZ13VvkNkRRJZaJYAjRPuN8sWlaoPtAOmGpmC4EDgYka0JYSbdwI118f\nprv+738wbhxccEHcUYnkvKSrx5bDdGCv6NraS4B+wOmFK939e6BJ4X0zmwr80d1VyEmKd9558Oij\ncOaZcOed0Lhx3BGJVAkpSxTunh9Vmp0EVAcecvc5ZnYTMMPdJ6bqtSWHrFsXyoHvuCNcdRX07Qu9\nesUdlUiVUmr12Eyk6rFVyH/+A+eeC127wpgxcUcjktUqUj1WZ2ZL5lm9OlR5PfJIqFFD4xAiMUvl\nGIXItnvrrdC9tGwZ/OlPYfC6bt24oxKp0pQoJLM0axbOhXj++TC7SURip64niZc7PP44nHVWuN2i\nRWhVKEmIZAwlConPV1+FK8yddRZ8+SX88EPZjxGRtFOikPQrKID77oO2beH11+Guu+DNN6FBg7gj\nE5FiaIxC0m/lynAhoQMPhFGjoGXLuCMSkVKoRSHpkZ8fzqouKICddoLp02HyZCUJkSygRCGp99FH\n4aS5AQPgxRfDsj33VBE/kSyhRCGpk5cH114LXbrAkiUwfjwcf3zcUYnINtIYhaTO8ceHMhxnnw13\n3AGNGsUdkYiUg2o9SeVauxZq14aaNWHSpHBuRM+ecUclUuWp1pNkhsmToV07GDYs3D/mGCUJkRyg\nRCEV9913cM45ITHUqQOHHhp3RCJSiZQopGJefhnatIF//hOuvhpmzYKDD447KhGpRBrMloqpXRua\nNoWXXoL99487GhFJASUK2Tbu8NhjsGhRmPp6+OHh5LlqapyK5Cr9d0vyFi4Mg9MDBsCrr4azrUFJ\nQiTH6T9cylZQAHffHWY0vfMO3HMPTJkSrj4nIjlP/+lStjlz4LLL4KijYORI2GOPuCMSkTRSi0KK\nt2lTGKAGaN8e3n8/3FeSEKlylCjk5z78EA44AHr1gk8+Ccs6d1YRP5EqSolCttqwIZwL8atfwTff\nwL/+FVoTIlKlaYxCgvz8cCGhjz+G3/0Obr8ddtwx7qhEJAMoUVR1eXmh7EaNGvCHP4TrRBx5ZNxR\niUgGUddTVfbyy7DPPjBhQrh/3nlKEiLyM0oUVdHKleEaEcceC9tvD7/4RdwRiUgGU6Koap59NhTx\ne/JJuO66MMOpa9e4oxKRDKYxiqpm8WJo3jxcO2K//eKORkSygK5wl+vc4ZFHQhdT376hHEdBgcpv\niFQxusKdFG/BAjj66DDddcyYsKxaNSUJEdkmShS5aPNmuOuuUMTvvffgvvvgmWfijkpEspS+Wuai\nF18MRfx69YL77w9jEiIi5aQWRa748cdwASGA448Pg9UvvKAkISIVltJEYWY9zexzM5tnZkOKWX+5\nmc01s4/N7D9mptKk5TFjRqjPdPjhsHx5KN531FEq4icilSJlicLMqgMjgGOBNkB/M2tTZLMPgS7u\n3gEYD9yWqnhy0oYNcNVV4TyIFSvCuRE77RR3VCKSY1I5RnEAMM/d5wOY2VigDzC3cAN3fy1h+2nA\nmSmMJ7esWhVKgc+bB+eeC7fdBg0bxh2ViOSgVCaKpsCihPuLgdJOAR4IvFTcCjMbBAwCqF27Q2XF\nl502b4bq1UNl1z59woD1EUfEHZWI5LCMGMw2szOBLsCw4ta7+yh37+LuXWrWrJne4DLJiy9C69bw\n6afh/u2/oRD3AAAOZUlEQVS3K0mISMqlMlEsARKn3DSLlv2EmR0JXAOc4O4bUxhP9lqxAs48E447\nDmrVgo3aTSKSPqlMFNOBvcyspZnVAvoBExM3MLP9gZGEJLEshbFkr3HjQhG/p56C66+HDz6Ajh3j\njkpEqpCUjVG4e76ZXQRMAqoDD7n7HDO7CZjh7hMJXU31gKctTOX8yt1PSFVMWWnKFGjRAh58UJcl\nFZFYqChgpnGH0aNh//2hSxdYvx5q1w4D2CIi5aSigLniyy+hRw8YNAgeeigs2247JQkRiZUSRSbY\nvBnuuCN0Lc2cCaNGwYgRcUclIgKoKGBmuOceuOIK6N07VHpt2jTuiEREtlCiiMuPP8KiRbDnnuHM\n6ubN4aSTVJ9JRDKOup7iMH06dO4MxxwTEsZ228HJJytJiEhGUqJIp/Xr4Y9/hAMPDLWa7rwznEAn\nIpLB1PWULvPnh9Lf8+fDeefB0KHQoEHcUYmIlEmJItXcQ5dS8+bh3IgHH4TDDos7KhGRpKnrKZWe\nfz5cK2L1aqhZE8aPV5IQkayjRJEKy5dD//5wwgmQlwfLVMZKRLKXEkVlcg9Xmdt3X3jmGbjppnCZ\n0r33jjsyEZFy0xhFZXKHe++FVq3CWETbtnFHJCJSYUoUFVVQEJLCCSfALrvAs89Co0aqzyQiOUNd\nTxXx3/+GK8wNGgQPPBCW7bSTkoSI5BQlivLIz4dhw6BDB5g1K5QFv+aauKMSEUkJdT2Vx1VXwfDh\n0KdPGJPYbbe4IxLJSJs2bWLx4sXk5eXFHUqVUadOHZo1a0bNmjUr7TmVKJK1cSP88EPoWrrsslCG\n49RTVZ9JpBSLFy+mfv36tGjRAtP/Ssq5OytXrmTx4sW0bNmy0p5XXU/JmDYNOnWCM88MM5t23x36\n9lWSEClDXl4ejRs3VpJIEzOjcePGld6CU6Iozbp1MHgwHHQQrFkTWhL6wItsEyWJ9ErF/lbXU0lm\nzQqlvxcsgAsvhFtvhfr1445KRCTt1KIoyW67wS9+AW+8Ea5ApyQhkrWee+45zIzPPvtsy7KpU6dy\n/PHH/2S7AQMGMH78eCAMxA8ZMoS99tqLTp060a1bN1566aUKx3LrrbfSqlUr9tlnHyZNmlTsNlOm\nTKFTp060a9eOs88+m/z8fAC+//57evfuzX777Ufbtm15+OGHKxxPMpQoEj33HJxySriG9c47wzvv\nwCGHxB2ViFTQmDFjOPjggxkzZkzSj7nuuutYunQps2fP5oMPPuC5555jzZo1FYpj7ty5jB07ljlz\n5vDyyy/zhz/8gc2bN/9km4KCAs4++2zGjh3L7Nmz2WOPPXj00UcBGDFiBG3atOGjjz5i6tSpXHHF\nFfz4448ViikZ6noC+PZbuPhiePrpcG7EsmWw665xRyWSUy67LPToVqaOHcP1v0qzdu1a3nrrLV57\n7TV69+7NjTfeWObzrl+/ngceeIAFCxZQu3ZtAHbZZRf69u1boXgnTJhAv379qF27Ni1btqRVq1a8\n//77dOvWbcs2K1eupFatWuwd1Yg76qijuPXWWxk4cCBmxpo1a3B31q5dS6NGjahRI/WH8ardonCH\nxx4LRfwmTICbbw5F/JQkRHLGhAkT6NmzJ3vvvTeNGzdm5syZZT5m3rx57L777uywww5lbjt48GA6\nduz4s5+//e1vP9t2yZIlNG/efMv9Zs2asWTJkp9s06RJE/Lz85kxYwYA48ePZ9GiRQBcdNFFfPrp\np+y22260b9+eu+66i2rVUn8Yr9otinXr4OqrQ6IYPTr8FpGUKOubf6qMGTOGSy+9FIB+/foxZswY\nOnfuXOLsoG2dNTR8+PAKx1j09ceOHcvgwYPZuHEjRx99NNWjskCTJk2iY8eOTJkyhS+//JKjjjqK\nQw45JKmEVhFVL1EUFMATT8Bpp0G9evDmm7DHHqrPJJKDvvvuO6ZMmcInn3yCmbF582bMjGHDhtG4\ncWNWrVr1s+2bNGlCq1at+Oqrr/jhhx/KPAgPHjyY11577WfL+/Xrx5AhQ36yrGnTpltaBxBOSGza\ntOnPHtutWzfefPNNACZPnswXX3wBwMMPP8yQIUMwM1q1akXLli357LPPOOCAA5LbIeXl7ln1U69e\nZy+3zz93P+QQd3B/9NHyP4+IJGXu3Lmxvv7IkSN90KBBP1l26KGH+uuvv+55eXneokWLLTEuXLjQ\nd999d1+9erW7u1955ZU+YMAA37hxo7u7L1u2zJ966qkKxTN79mzv0KGD5+Xl+fz5871ly5aen5//\ns+2+/fZbd3fPy8vzI444wv/zn/+4u/v555/v119/vbu7f/PNN77bbrv58uXLf/b44vY7MMPLedyt\nGmMU+fkwdGgYqP7kE3j4YTjrrLijEpEUGzNmDCeddNJPlp1yyimMGTOG2rVr8/jjj3POOefQsWNH\nfvOb3zB69GgaNGgAwM0338xOO+1EmzZtaNeuHccff3yFu3jatm1L3759adOmDT179mTEiBFbupV6\n9erF119/DcCwYcPYd9996dChA7179+aII44Awkysd955h/bt29OjRw+GDh1KkyZNKhRTMiwkmuxR\nv34XX7NmxrY96LTT4Kmnwgl0I0aE8yNEJOU+/fRT9tXYX9oVt9/NbKa7dynP8+XuGEVeXpjVVLdu\nmPrat284R0JERLZJbnY9vf12mGB97bXh/sEHK0mIiJRTbiWKtWvhkkvC2dR5eXDMMXFHJFLlZVv3\ndrZLxf7OnUTx1lvQrl2oy3TRRTB7Nhx9dNxRiVRpderUYeXKlUoWaeLR9Sjq1KlTqc+bO2MU220X\nCve9+Sb8+tdxRyMihDOPFy9ezPLly+MOpcoovMJdZcruWU//+hdMnx5KgEM4mS4Np7OLiGSbisx6\nSulR1cx6mtnnZjbPzIYUs762mY2L1r9nZi2SeuJvvoHf/CYMUE+eDBs2hOVKEiIilS5lR1Yzqw6M\nAI4F2gD9zaxNkc0GAqvcvRUwHBha1vM22LQy1GR64YXQkpg2LUyBFRGRlEjlV/ADgHnuPt/dfwTG\nAn2KbNMHeDS6PR7oYWVU5Npl4//CoPVHH8GQIVCzZqUHLiIiW6VyMLspsCjh/mKga0nbuHu+mX0P\nNAZWJG5kZoOAQdHdjfbWW7Np3TolQWeZJhTZV1WY9sVW2hdbaV9stU95H5gVs57cfRQwCsDMZpR3\nQCbXaF9spX2xlfbFVtoXW5nZNtY+2iqVXU9LgOYJ95tFy4rdxsxqAA2AlSmMSUREtlEqE8V0YC8z\na2lmtYB+wMQi20wEzo5u/waY4tk2X1dEJMelrOspGnO4CJgEVAcecvc5ZnYToS76ROBB4J9mNg/4\njpBMyjIqVTFnIe2LrbQvttK+2Er7Yqty74usO+FORETSS2eoiYhIqZQoRESkVBmbKFJW/iMLJbEv\nLjezuWb2sZn9x8z2iCPOdChrXyRsd4qZuZnl7NTIZPaFmfWNPhtzzOzJdMeYLkn8j+xuZq+Z2YfR\n/0mvOOJMNTN7yMyWmdnsEtabmf0j2k8fm1mnpJ64vBfbTuUPYfD7S+CXQC3gI6BNkW3+ANwf3e4H\njIs77hj3xeHAdtHtC6ryvoi2qw+8AUwDusQdd4yfi72AD4Edo/s7xx13jPtiFHBBdLsNsDDuuFO0\nLw4FOgGzS1jfC3gJMOBA4L1knjdTWxQpKf+RpcrcF+7+mruvj+5OI5yzkouS+VwA/IVQNywvncGl\nWTL74lxghLuvAnD3ZWmOMV2S2RcO7BDdbgB8ncb40sbd3yDMIC1JH+AxD6YBDc1s17KeN1MTRXHl\nP5qWtI275wOF5T9yTTL7ItFAwjeGXFTmvoia0s3d/d/pDCwGyXwu9gb2NrO3zWyamfVMW3Tplcy+\nuAE408wWAy8CF6cntIyzrccTIEtKeEhyzOxMoAvQPe5Y4mBm1YA7gAExh5IpahC6nw4jtDLfMLP2\n7r461qji0R94xN3/bmbdCOdvtXP3grgDywaZ2qJQ+Y+tktkXmNmRwDXACe6+MU2xpVtZ+6I+0A6Y\namYLCX2wE3N0QDuZz8ViYKK7b3L3BcAXhMSRa5LZFwOBpwDc/V2gDqFgYFWT1PGkqExNFCr/sVWZ\n+8LM9gdGEpJErvZDQxn7wt2/d/cm7t7C3VsQxmtOcPdyF0PLYMn8jzxHaE1gZk0IXVHz0xlkmiSz\nL74CegCY2b6ERFEVr886EfhtNPvpQOB7d19a1oMysuvJU1f+I+skuS+GAfWAp6Px/K/c/YTYgk6R\nJPdFlZDkvpgEHG1mc4HNwJXunnOt7iT3xRXAA2Y2mDCwPSAXv1ia2RjCl4Mm0XjM9UBNAHe/nzA+\n0wuYB6wHzknqeXNwX4mISCXK1K4nERHJEEoUIiJSKiUKEREplRKFiIiUSolCRERKpUQhApjZZjOb\nlfDTopRtWxRW5zSzw8zshUqK4TAzO6gynkukMmXkeRQiMdjg7h1jjuEwYC3wTsxxiPyEWhQiJYha\nDm+a2QfRzzZ92zezHtH1Dz6JrhNQO1q+MDpTGjPrYmZToxbM+cDgqEVzSGW/H5HyUqIQCeomdDs9\nGy1bBhzl7p2A04B/JPtkZlYHeAQ4zd3bE1rvF5S0vbsvBO4Hhrt7R3d/s3xvQ6TyqetJJCiu66km\ncI+ZdSSUwNh7G55vH2CBu38R3X8UuBC4s8KRiqSZEoVIyQYD3wL7EVrfpV4IycwmAbsAM4C7S9k0\nn62t+ToVD1MktZQoRErWAFjs7gVmdjah4FyJ3P2YwttR11MLM2vl7vOAs4DXo9ULgc6EC0ydkvAU\na9h6FTaRjKExCpGS3QucbWYfAa2Bdck+0N3zCJU5nzazT4ACwhgEwI3AXWY2g9ClVeh54CQNZkum\nUfVYEREplVoUIiJSKiUKEREplRKFiIiUSolCRERKpUQhIiKlUqIQEZFSKVGIiEip/h+SIN/PzsnC\nLQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f391c49b780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions = classifier.predict_proba(X_test)\n",
    "false_positive_rate, recall, thresholds = roc_curve(y_test, predictions[:, 1])\n",
    "roc_auc = auc(false_positive_rate, recall)\n",
    "plt.title('Receiver Operating Characteristic')\n",
    "plt.plot(false_positive_rate, recall, 'b', label='AUC = %0.2f' % roc_auc)\n",
    "plt.legend(loc='lower right')\n",
    "plt.plot([0, 1], [0, 1], 'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.0])\n",
    "plt.ylabel('Recall')\n",
    "plt.xlabel('Fall-out')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
